/*
 *  Copyright 2005 Blandware (http://www.blandware.com)
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.blandware.atleap.webapp.util.core;

import com.blandware.atleap.common.Constants;
import com.blandware.atleap.common.util.ConvertUtil;
import com.blandware.atleap.common.util.RegExUtil;
import com.blandware.atleap.service.core.StatisticsManager;
import com.blandware.atleap.service.core.util.RawData;
import com.blandware.atleap.service.core.util.VisitorSoftwareUtil;
import org.apache.commons.validator.GenericValidator;
import org.apache.oro.text.regex.*;
import org.springframework.context.ApplicationContext;
import org.springframework.web.context.support.WebApplicationContextUtils;

import javax.servlet.ServletContext;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import java.util.*;
import java.text.ParseException;

/**
 * <p>Contains utils used in web layer of statistics module. Also caches some
 * values (next visitor number, next session number...).</p>
 * <p><a href="StatisticsUtil.java.html"><i>View source</i></a></p>
 *
 * @author Roman Puchkovskiy <a href="mailto:roman.puchkovskiy@blandware.com">
 *         &lt;roman.puchkovskiy@blandware.com&gt;</a>
 * @version $Revision: 1.13 $ $Date: 2007/12/20 13:41:54 $
 */
public class StatisticsUtil {

	/**
	 * Servlet context key under which instance of this class is saved
	 */
	private static final String INSTANCE_KEY = "com.blandware.atleap.webapp.util.StatisticsUtil.INSTANCE";

    /**
     * Regex for IP
     */
    private static final String IP_REGEX = "^(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$";

    /**
     * Max value for unsigned byte
     */
    private static final int MAX_UBYTE = 255;

    /**
     * Statistics manager we are using
     */
    private StatisticsManager statisticsManager = null;

    /**
     * Contains a visitor number that will be returned next time
     */
    protected Integer nextVisitorNumber = new Integer(0);

	/**
	 * Creates new instance of StatisticsUtil and initializes internal fields.
	 *
	 * @param servletContext Servlet context
	 */
	private StatisticsUtil(ServletContext servletContext) {
		ApplicationContext applicationContext = WebApplicationContextUtils.getRequiredWebApplicationContext(servletContext);
		statisticsManager = (StatisticsManager) applicationContext.getBean(Constants.STATISTICS_MANAGER_BEAN);
        initialize();
	}

	/**
	 * Returns instance of StatisticsUtil corresponding to a given servlet
     * context.
	 *
     * @param servletContext Servlet context
	 * @return Instance of StatisticsUtil
	 */
	public static synchronized StatisticsUtil getInstance(ServletContext servletContext) {
		StatisticsUtil statisticsUtil = (StatisticsUtil) servletContext.getAttribute(INSTANCE_KEY);
		if ( statisticsUtil == null ) {
			statisticsUtil = new StatisticsUtil(servletContext);
			servletContext.setAttribute(INSTANCE_KEY, statisticsUtil);
		}
		return statisticsUtil;
	}

    /**
     * Initializes StatisticsUtil instance.
     */
    private void initialize() {
        synchronized (this) {
            nextVisitorNumber = statisticsManager.getFirstFreeVisitorNumber();
        }
    }

    /**
     * Serves a visit: extracts raw data and, if needed, saves it to collector.
     *
     * @param request request from which to get data
     * @param response response we are creating
     * @param extractFromParams whether such things as URL and referrer URL
     * should be taken from request parameters, not from headers
     */
    public static void serveVisit(HttpServletRequest request,
                                  HttpServletResponse response,
                                  boolean extractFromParams) {
        HttpSession session = request.getSession();
        ServletContext servletContext = session.getServletContext();

        Integer visitorNumber = StatisticsUtil.getVisitorNumberFromCookie(request);

        ApplicationContext ctx = WebApplicationContextUtils.getRequiredWebApplicationContext(servletContext);
        StatisticsManager statisticsManager = (StatisticsManager) ctx.getBean(Constants.STATISTICS_MANAGER_BEAN);
        VisitorSoftwareUtil visitorSoftwareUtil = (VisitorSoftwareUtil) ctx.getBean(Constants.VISITOR_SOFTWARE_UTIL_BEAN);
        StatisticsUtil statisticsUtil = StatisticsUtil.getInstance(servletContext);
        GlobalProperties globalProperties = GlobalProperties.getInstance(servletContext);

        String forbiddenPaths = globalProperties.getString(Constants.STATISTICS_FORBIDDEN_PATHS);
        String forbiddenIpsAndSubnets = globalProperties.getString(Constants.STATISTICS_FORBIDDEN_IPS_AND_SUBNETS);
        RawData data = StatisticsUtil.extractRawData(request,
                                                     extractFromParams);

        if (needsToBeConsidered(data, visitorSoftwareUtil, forbiddenPaths,
                                forbiddenIpsAndSubnets, extractFromParams)) {
            Integer flushThreshold = globalProperties.getInteger(Constants.STATISTICS_FLUSH_THRESHOLD);
            RawDataCollector collector = RawDataCollector.getInstance(session,
                    visitorNumber, statisticsUtil, flushThreshold);
            collector.addData(data, statisticsManager);
            StatisticsUtil.setStatisticsCookie(request, response,
                                               collector.getVisitorNumber());
        }
    }

    /**
     * Figures out whether this visit needs to be considered or we must ignore
     * it.
     *
     * @param data raw data
     * @param visitorSoftwareUtil used to determine whether this is robot
     * @param forbiddenPaths comma-separated list of regular expressions for
     * paths on which statistics data is not collected
     * @param forbiddenIpsAndSubnets comma-separated list of IPs and subnet
     * expressions for IPs and subnets for which statistics data is not
     * collected
     * @param extractFromParams whether data was extracted from params
     * @return whether this visit needs to be considered
     */
    protected static boolean needsToBeConsidered(RawData data,
                                                 VisitorSoftwareUtil visitorSoftwareUtil,
                                                 String forbiddenPaths,
                                                 String forbiddenIpsAndSubnets,
                                                 boolean extractFromParams) {
        boolean skip = false;

        if (GenericValidator.isBlankOrNull(data.getUrl())) {
            // empty URL is not interesting
            skip = true;
        }

        if (!skip && data.isNotResource() && !extractFromParams) {
            // data came from filter, not from resource: this needs to be
            // considered only if this is robot
            String userAgent = data.getUserAgent();
            if (userAgent == null) {
                // no user-agent at all, this is not a robot
                skip = true;
            } else {
                Integer robotId = visitorSoftwareUtil.detectRobotIdByUserAgentString(userAgent);
                if (Constants.UNKNOWN_SOFTWARE_ID.equals(robotId)) {
                    // this is not a robot
                    skip = true;
                }
            }
        }

        if (!skip) {
            // checking path
            List regexes = ConvertUtil.convertStringToList(forbiddenPaths, ",",
                                                           true);
            for (Iterator i = regexes.iterator(); i.hasNext(); ) {
                String regex = (String) i.next();
                boolean matches;
                try {
                    matches = RegExUtil.matches(data.getPath(), regex);
                } catch (MalformedPatternException e) {
                    matches = false;
                }
                if (matches) {
                    // some regex matched
                    skip = true;
                    break;
                }
            }
        }

        if (!skip) {
            // checking IP
            if (ipIsForbidden(data.getIp(), forbiddenIpsAndSubnets)) {
                skip = true;
            }
        }

        return !skip;
    }

    /**
     * <p>Returns whether IP should be ignored if it matches to some expression
     * from list of forbidden IPs and subnets. Each expression is IP or subnet:
     * IP/number, where number is an integer from 0 to 32.</p>
     * <p>IPs should be in IPv4 format.</p>
     *
     * @param ip IP to check
     * @param forbiddenIpsAndSubnets comma-separated list of expressions
     * @return whether IP should be ignored
     */
    protected static boolean ipIsForbidden(String ip,
                                           String forbiddenIpsAndSubnets) {
        boolean result;
        if (GenericValidator.isBlankOrNull(ip)) {
            // empty IP is not interesting: actually, this is incorrect, so
            // ignore it
            result = true;
        } else {
            result = false;
            List expressionsList = ConvertUtil.convertStringToList(forbiddenIpsAndSubnets,
                                                                   ",", true);
            for (Iterator i = expressionsList.iterator(); i.hasNext(); ) {
                String expression = (String) i.next();
                if (ipMatchesExpression(ip, expression)) {
                    result = true;
                    break;
                }
            }
        }
        return result;
    }

    /**
     * <p>Returns true if IP matches expression. Expression is IP or subnet:
     * IP/number, where number is an integer from 0 to 32. If it's IP, given IP
     * is just compared to it. If it's a subnet, given IP is checked whether it
     * belongs to this subnet.</p>
     * <p>IPs should be in IPv4 format.</p>
     *
     * @param ip IP to check
     * @param expression IP or subnet
     * @return whether IP matches expression
     */
    protected static boolean ipMatchesExpression(String ip, String expression) {
        boolean result = false;

        if (GenericValidator.isBlankOrNull(ip)) {
            return false;
        }
        if (GenericValidator.isBlankOrNull(expression)) {
            return false;
        }

        boolean valid = true;
        int slash = expression.indexOf('/');
        boolean isSubnet = slash != -1;
        String targetIp;
        String maskLenStr;
        int maskLen = 0;

        if (isSubnet) {
            targetIp = expression.substring(0, slash);
            maskLenStr = expression.substring(slash + 1);
            if (GenericValidator.isBlankOrNull(maskLenStr)) {
                valid = false;
            }
            if (valid) {
                try {
                    maskLen = Integer.parseInt(maskLenStr);
                    if (maskLen < 0 || maskLen > 32) {
                        valid = false;
                    }
                } catch (NumberFormatException e) {
                    valid = false;
                }
            }
        } else {
            targetIp = expression;
            maskLen = 32;
        }
        if (valid && GenericValidator.isBlankOrNull(targetIp)) {
            valid = false;
        }

        if (valid) {
            try {
                int ipAsInt = parseIpAsInt(ip);
                int targetAsInt = parseIpAsInt(targetIp);
                int mask = buildMask(maskLen);
                result = (ipAsInt & mask) == (targetAsInt & mask);
            } catch (ParseException e) {
                valid = false;
            }
        }

        if (!valid) {
            result = false;
        }

        return result;
    }

    /**
     * Parses an IP (as IPv4) and constructs int: its highest byte is same as
     * first IP byte and so on.
     *
     * @param ip IP to parse
     * @return result
     * @throws ParseException thrown if IP has incorrect format
     */
    protected static int parseIpAsInt(String ip) throws ParseException {
        int result = 0;
        boolean ok = true;

        String regex = IP_REGEX;
        PatternCompiler compiler = new Perl5Compiler();
        PatternMatcher matcher = new Perl5Matcher();
        Pattern pattern;
        try {
            pattern = compiler.compile(regex);
        } catch (MalformedPatternException e) {
            pattern = null;
        }
        boolean matches = matcher.matches(ip, pattern);
        if (matches) {
            MatchResult matchResult = matcher.getMatch();
            String fragment1 = matchResult.group(1);
            String fragment2 = matchResult.group(2);
            String fragment3 = matchResult.group(3);
            String fragment4 = matchResult.group(4);
            int byte1 = Integer.parseInt(fragment1);
            int byte2 = Integer.parseInt(fragment2);
            int byte3 = Integer.parseInt(fragment3);
            int byte4 = Integer.parseInt(fragment4);
            if (byte1 > MAX_UBYTE || byte2 > MAX_UBYTE
                    || byte3 > MAX_UBYTE || byte4 > MAX_UBYTE) {
                ok = false;
            }
            if (ok) {
                result = byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4;
            }
        } else {
            ok = false;
        }

        if (ok) {
            return result;
        } else {
            throw new ParseException("Bad IP format", 0);
        }
    }

    /**
     * Builds a mask for subnet. Returns int where maskLen highest bits are 1,
     * the rest are 0.
     *
     * @param maskLen how much high bits to set to 1
     * @return mask
     */
    protected static int buildMask(int maskLen) {
        int result = 0x0;
        for (int i = 0, curMask = 1 << 31; i < maskLen; i++, curMask >>= 1) {
            result |= curMask;
        }
        return result;
    }

    /**
     * Extracts raw data from a request and returns it.
     *
     * @param request request from which to extract raw data
     * @param extractFromParams whether such things as URL and referrer URL
     * should be taken from request parameters, not from headers
     * @return extracted data
     */
    public static RawData extractRawData(HttpServletRequest request,
                                         boolean extractFromParams) {
        String url;
        String referrerUrl;

        RawData data = new RawData();

        data.setDate(new Date());
        if (extractFromParams) {
            String probableUrl = request.getParameter(WebappConstants.URL_PARAM);
            if (GenericValidator.isBlankOrNull(probableUrl)) {
                probableUrl = request.getHeader(WebappConstants.REFERRER_HEADER);
            }
            url = probableUrl;
            referrerUrl = request.getParameter(WebappConstants.REFERRER_PARAM);
            data.setReferrerUrl(referrerUrl);
        } else {
            url = request.getRequestURL().toString();
            referrerUrl = request.getHeader(WebappConstants.REFERRER_HEADER);
            data.setReferrerUrl(referrerUrl);
        }
        data.setIp(request.getRemoteAddr());
        data.setUserName(request.getRemoteUser());
        data.setUserAgent(request.getHeader(WebappConstants.USER_AGENT_HEADER));
        data.setScreenInfo(request.getParameter(WebappConstants.SCREEN_INFO_PARAM));

        String protocol = null;
        String domain = null;
        String port = null;
        String path = null;
        String queryString = null;
        String queryStringAndQuestion = null;
        Map queryParameterMap = new HashMap();
        String referrerProtocol = null;
        String referrerDomain = null;
        String referrerPort = null;
        String referrerPortAndColon = null;
        String referrerPath = null;
        String referrerQueryString = null;
        String referrerQueryStringAndQuestion = null;
        Map referrerQueryParameterMap = new HashMap();

        String regex = "^(https?:\\/\\/)?([a-zA-Z0-9\\.\\-_]+)(:(\\d+))?(\\/[^?]*)?(\\?(.*))?$";
        PatternCompiler compiler = new Perl5Compiler();
        Pattern pattern = null;
        try {
            pattern = compiler.compile(regex);
        } catch (MalformedPatternException e) {
            // actually can happen only if we screwed regex
            throw new RuntimeException(e);
        }

        if (!GenericValidator.isBlankOrNull(url)) {
            PatternMatcher matcher = new Perl5Matcher();
            boolean matches = matcher.matches(url, pattern);
            if (matches) {
                MatchResult matchResult = matcher.getMatch();
                protocol = matchResult.group(1);
                domain = matchResult.group(2);
                port = matchResult.group(4);
                path = matchResult.group(5);
                queryStringAndQuestion = matchResult.group(6);
                queryString = matchResult.group(7);
            }
        }

        String contextPath = request.getContextPath();
        String contextlessPath = path;
        if (!GenericValidator.isBlankOrNull(path)) {
            // remove context path
            if ( contextlessPath.startsWith(contextPath) && !contextPath.equals("/") ) {
                contextlessPath = contextlessPath.substring(contextPath.length());
            }
        }

        if (GenericValidator.isBlankOrNull(path)) {
            path = "/";
        }
        if (GenericValidator.isBlankOrNull(contextlessPath)) {
            contextlessPath = "/";
        }

        data.setUrl((contextlessPath == null ? "" : contextlessPath)
                    + (queryStringAndQuestion == null
                       ? "" : queryStringAndQuestion));

        if (!GenericValidator.isBlankOrNull(referrerUrl)) {
            PatternMatcher matcher = new Perl5Matcher();
            boolean matches = matcher.matches(referrerUrl, pattern);
            if (matches) {
                MatchResult matchResult = matcher.getMatch();
                referrerProtocol = matchResult.group(1);
                referrerDomain = matchResult.group(2);
                referrerPortAndColon = matchResult.group(3);
                referrerPort = matchResult.group(4);
                referrerPath = matchResult.group(5);
                referrerQueryStringAndQuestion = matchResult.group(6);
                referrerQueryString = matchResult.group(7);

                if (GenericValidator.isBlankOrNull(referrerPath)) {
                    String newReferrerPath = "/";
                    String newReferrerUrl = (referrerProtocol == null ? "" : referrerProtocol)
                            + (referrerDomain == null ? "" : referrerDomain)
                            + (referrerPortAndColon == null ? "" : referrerPortAndColon)
                            + newReferrerPath
                            + (referrerQueryStringAndQuestion == null ? "" : referrerQueryStringAndQuestion);
                    data.setReferrerUrl(newReferrerUrl);
                }
            }
        }

        HttpSession session = request.getSession();
        ServletContext servletContext = session.getServletContext();
        String objectUri = WebappUtil.extractObjectUri(url,
                                                       servletContext,
                                                       contextPath,
                                                       request);
        String pageUri = null;
        String contentResourceUri = null;
        if (objectUri != null) {
            if (objectUri.startsWith(Constants.RESOURCES_URI_PREFIX)) {
                contentResourceUri = objectUri;
            } else {
                pageUri = objectUri;
            }
        }
        data.setPageUri(pageUri);
        data.setContentResourceUri(contentResourceUri);

        data.setPath(path);
        data.setReferrerDomain(referrerDomain);
        if (!GenericValidator.isBlankOrNull(referrerQueryString)) {
            referrerQueryParameterMap = RequestUtil.getRequestParametersFromString(referrerQueryString, false);
        }
        data.setReferrerQueryParameters(referrerQueryParameterMap);

        return data;
    }

    /**
     * Returns visitor number from cookie.
     *
     * @param request request from which to take cookie
     * @return extracted visitor number or <code>null</code> if no found
     */
    public static Integer getVisitorNumberFromCookie(HttpServletRequest request) {
        Integer result = null;
        Cookie cookie = RequestUtil.getCookie(request,
                                              WebappConstants.STATISTICS_COOKIE);
        if (cookie != null) {
            String value = cookie.getValue();
            if (!GenericValidator.isBlankOrNull(value)) {
                try {
                    result = Integer.valueOf(value);
                } catch (NumberFormatException e) {
                    result = null;
                }
            }
        }
        return result;
    }

    /**
     * Sets a statistics cookie.
     *
     * @param request request which is currently processed
     * @param response response to which to put cookie
     * @param visitorNumber visitor number
     */
    public static void setStatisticsCookie(HttpServletRequest request,
                                           HttpServletResponse response,
                                           Integer visitorNumber) {
        String path = request.getContextPath();
        if (GenericValidator.isBlankOrNull(path)) {
            path = "/";
        }

        RequestUtil.setCookie(response, WebappConstants.STATISTICS_COOKIE,
                              visitorNumber.toString(), path);
    }

    /**
     * Returns the least of numbers that were not ever used as a visitor number.
     *
     * @return first free visitor number
     */
    public Integer getNextFreeVisitorNumber() {
        synchronized (this) {
            Integer result = nextVisitorNumber;
            nextVisitorNumber = new Integer(nextVisitorNumber.intValue() + 1);
            return result;
        }
    }

    /**
     * Returns whether visitor number is correct.
     *
     * @param visitorNumber visitor number to check
     * @return whether visitor number is correct
     */
    public boolean visitorNumberIsCorrect(Integer visitorNumber) {
        synchronized (this) {
            return visitorNumber.intValue() < nextVisitorNumber.intValue();
        }
    }

}
